package com.zhan.ielts.tools;


import net.htmlparser.jericho.*;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;


import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.sql.*;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by Admin on 2015/12/10.
 */
public class YouDaoFile {
    public static JSONObject parserHtml(String fileName){
        JSONObject result = new JSONObject();
        result.put("state","ok");
        File file = new File(fileName);
        if(!file.exists() || !file.isFile() || file.length()<=0){
            result.put("message","文件错误");
            result.put("state","error");
            return result;
        }
        Source source = null;
        try {
            Config.LoggerProvider= LoggerProvider.DISABLED;
            source = new Source(file);
        } catch (IOException e) {
            e.printStackTrace();
            result.put("message","读取文件失败");
            result.put("state","error");
            return result;
        }
        if(null == source){
            result.put("message","读取文件失败");
            result.put("state","error");
            return result;
        }
        JSONObject content = new JSONObject();
        List<StartTag> tagList = source.getAllStartTagsByClass("error-note");
        if(null != tagList && tagList.size()>0){
            Element el = tagList.get(0).getElement().getChildElements().get(0).getChildElements().get(0);
            String message = el.getContent().toString();
            result.put("message",filterSpecialChar(message));
            result.put("state","error");
            return result;
        }
        content.put("yd",_yd(source.getElementById("phrsListTab")));
//        content.put("radio",_youDaoRadio(source.getElementById("phrsListTab")));
        content.put("EE",_transEE(source.getElementById("tEETrans")));
        content.put("21",_21century(source.getElementById("authDictTrans")));
        content.put("gls",_collins(source.getElementById("collinsResult")));
        content.put("ph",_phrase(source));
        content.put("es",_examples(source.getElementById("bilingual")));
        content.put("bk",_baike(source.getElementById("bk")));
        result.put("content",content);
        return result;
    }
    public static JSONObject parserAudio(String fileName){
        JSONObject result = new JSONObject();
        result.put("state","ok");
        File file = new File(fileName);
        if(!file.exists() || !file.isFile() || file.length()<=0){
            result.put("message","文件错误");
            result.put("state","error");
            return result;
        }
        Source source = null;
        try {
            Config.LoggerProvider= LoggerProvider.DISABLED;
            source = new Source(file);
        } catch (IOException e) {
            e.printStackTrace();
            result.put("message","读取文件失败");
            result.put("state","error");
            return result;
        }
        if(null == source){
            result.put("message","读取文件失败");
            result.put("state","error");
            return result;
        }
        JSONObject content = new JSONObject();
        List<StartTag> tagList = source.getAllStartTagsByClass("error-note");
        if(null != tagList && tagList.size()>0){
            Element el = tagList.get(0).getElement().getChildElements().get(0).getChildElements().get(0);
            String message = el.getContent().toString();
            result.put("message",filterSpecialChar(message));
            result.put("state","error");
            return result;
        }
        content.put("radio",_youDaoRadio(source.getElementById("phrsListTab")));
        result.put("content",content);
        return result;
    }
    public static JSONObject _yd(Element element){
        JSONObject yd = new JSONObject();
        if(null == element || element.isEmpty()){
            return null;
        }
        JSONArray pron = new JSONArray();
        List<StartTag> els = element.getAllStartTagsByClass("pronounce");
        if(null != els && els.size()>0){
            for (int i = 0; i < els.size(); i++) {
                JSONObject p = new JSONObject();
                String text = els.get(i).getElement().getContent().toString();
                if(text.indexOf("英")>-1){
                    p.put("uk",filterSpecialChar(els.get(i).getElement().getChildElements().get(0).getFirstElement().getContent().toString()));
                }else {
                    p.put("us",filterSpecialChar(els.get(i).getElement().getChildElements().get(0).getFirstElement().getContent().toString()));
                }
                pron.add(p);
            }
        }
        yd.put("pron",pron);
        JSONArray trans = new JSONArray();
        els = element.getAllStartTagsByClass("trans-container");
        if(null == els || !els.isEmpty()){
            List<Element> elist = els.get(0).getElement().getAllElements("li");
            if(null != elist && elist.size()>0) {
                for (int i = 0; i < elist.size(); i++) {
                    trans.add(filterSpecialChar(elist.get(i).getContent().toString()));
                }
            }
        }

        yd.put("trans",trans);
        List<Element> elist = element.getAllElementsByClass("additional");
        if(null != elist && elist.size()>0){
            Element el = element.getAllElementsByClass("additional").get(0);
            yd.put("additional",filterSpecialChar(el.getContent().toString()));
        }
        return yd;
    }

    public static JSONArray _youDaoRadio(Element element){
        JSONArray radio = new JSONArray();
        if(null == element || element.isEmpty()){
            return null;
        }
        List<Element> voice = element.getAllElementsByClass("voice-js");
        for (int j = 0; j < voice.size(); j++) {
            JSONObject p = new JSONObject();
            String url = voice.get(j).getAttributeValue("data-rel");
            p.put("url",url);
            radio.add(p);
        }
        return radio;
    }
    public static JSONObject _21century(Element element){
        JSONObject _21 = new JSONObject();
        if(null == element || element.isEmpty()){
            return null;
        }
        List<StartTag> els = element.getAllStartTagsByClass("phonetic");
        if(null != els && !els.isEmpty()){
            _21.put("pron",els.get(0).getElement().getContent().toString());
        }

        Element ul = element.getFirstElement("ul");
        JSONArray trans = getUlText(ul);
        _21.put("trans",trans);
        return _21;
    }

    public static JSONArray _collins(Element element){
        JSONArray collins = new JSONArray();
        if(null == element || element.isEmpty()){
            return null;
        }
        List<Element> div = element.getAllElementsByClass("wt-container");
        if(null == div || div.isEmpty()){
            return collins;
        }
        for (int i = 0; i < div.size(); i++) {
            JSONObject col = new JSONObject();
            JSONObject child = new JSONObject();
            JSONArray pron = new JSONArray();
            JSONArray trans = new JSONArray();
            Element h4 = null,titleEl = null;
            List<Element> ul = null;
            String key = null;
            if (null != div.get(i).getChildElements() && div.get(i).getChildElements().get(0).getName().equals("div")){
                titleEl = div.get(i).getFirstElementByClass("trans-tip");
                h4 = div.get(i).getFirstElementByClass("trans-content").getFirstElement("h4");
                ul = div.get(i).getFirstElementByClass("trans-content").getAllElements("ul");
            }else {
                h4 = div.get(i).getFirstElement("h4");
                ul = div.get(i).getAllElements("ul");
            }
            if(null != titleEl){
                key = filterSpecialChar(titleEl.getFirstElement("span").getContent().toString());
            }else {
                key = filterSpecialChar(h4.getFirstElementByClass("title").getContent().toString());
            }
            if (null != h4.getFirstElement("em") && !h4.getFirstElement("em").isEmpty()){
                pron.add(filterSpecialChar(h4.getFirstElement("em").getContent().toString()));
                col.put("pron",pron);
            }
            if (null != h4.getFirstElementByClass("via") && !h4.getFirstElementByClass("via").isEmpty()){
                col.put("level",filterSpecialChar(h4.getFirstElementByClass("via").getContent().toString()));
            }
            if (null != h4.getFirstElementByClass("pattern") && !h4.getFirstElementByClass("pattern").isEmpty()){
                col.put("additional",filterSpecialChar(h4.getFirstElementByClass("pattern").getContent().toString()));
            }
            if(null != ul && ul.size()>0){
                for (int j = 0; j < ul.size(); j++) {
                    trans.add(getElementText(ul.get(j),"b"));
                }
            }
            col.put("trans",trans);
            child.put(key,col);
            collins.add(child);
        }
        return collins;
    }

    public static JSONObject _phrase(Source source){
        JSONObject phrase = new JSONObject();
        Element wordGroup = source.getElementById("wordGroup");
        if(null != wordGroup && !wordGroup.isEmpty()){
            JSONArray wg = new JSONArray();
            List<Element> p = wordGroup.getAllElements("p");
            for (int i = 0; i < p.size(); i++) {
                JSONObject x = new JSONObject();
                String key = filterSpecialChar(p.get(i).getFirstElement("span").getContent().toString());
                String value = getText(p.get(i));
                x.put("cn",value);
                x.put("en",key);
                wg.add(x);
            }
            phrase.put("词组短语",wg);
        }
        Element synonyms = source.getElementById("synonyms");
        if(null != synonyms && !synonyms.isEmpty()){
            JSONArray s = new JSONArray();
            List<Element> li = synonyms.getFirstElement("ul").getChildElements();
            for (int i = 0; i < li.size();) {
                String key = filterSpecialChar(li.get(i).getContent().toString());
                String value = filterSpecialChar(li.get(i+1).getContent().toString());
                i += 2;
                JSONObject wg = new JSONObject();
                wg.put(key,value);
                s.add(wg);
            }
            phrase.put("同近义词",s);
        }
        Element relWordTab = source.getElementById("relWordTab");
        if(null != relWordTab && !relWordTab.isEmpty()){
            phrase.put("同根词",getElementText(relWordTab,"a,b"));
        }
        Element discriminate = source.getElementById("discriminate");
        if(null != discriminate && !discriminate.isEmpty()){
            List<Element> div = discriminate.getChildElements();
            JSONArray s = new JSONArray();
            for (int i = 0; i < div.size(); i++) {
                if (null == div.get(i) || !"div".equals(div.get(i).getName())) {
                    continue;
                }
                JSONObject wg = new JSONObject();
                Element keyEl = null,valueEl = null;
                keyEl = div.get(i).getFirstElementByClass("title");
                valueEl = div.get(i).getFirstElementByClass("collapse-content");
                if (null != keyEl && !keyEl.isEmpty()){
                    String key = getElementText(keyEl,"b");
                    String value = getElementText(valueEl,"b");
                    wg.put(key,value);
                    s.add(wg);
                }
            }
            phrase.put("词语辨析",s);
        }
        if(phrase.isEmpty()){
            return null;
        }
        return phrase;
    }

    public static JSONArray _examples(Element element){
        JSONArray examples = new JSONArray();
        if(null == element || element.isEmpty()){
            return null;
        }
        Element ul = element.getFirstElement("ul");
        if(null == ul || ul.isEmpty()){
            return null;
        }
        List<Element> li = ul.getChildElements();
        for (int i = 0; i < li.size(); i++) {
            JSONObject x = new JSONObject();
            List<Element> cs = li.get(i).getChildElements();
            if(null == cs || cs.size()<=0) continue;
            x.put("en", getElementText(cs.get(0),"span,a,b"));
            x.put("cn", getElementText(cs.get(1),"span,a,b"));
            examples.add(x);
        }
        return examples;
    }

    public static String _baike(Element element){
        String bk = null;
        if(null == element || element.isEmpty()){
            return null;
        }
        Element value = element.getFirstElementByClass("content");
        if(null == value || value.isEmpty()){
            return null;
        }
        bk = getElementText(value,"a");
        return bk;
    }

    public static JSONObject _transEE(Element element){
        JSONObject EE = new JSONObject();
        if(null == element || element.isEmpty()){
            return null;
        }
        Element ul = element.getFirstElement("div").getFirstElement("ul");
        JSONArray trans = getUlText(ul);
        EE.put("trans",trans);
        return EE;
    }

    public static JSONArray getUlText(Element el){
        JSONArray UL = new JSONArray();
        if(null != el && !el.isEmpty()){
            List<Element> li = el.getChildElements();
            if (null != li && !li.isEmpty()){
                for (int i = 0; i < li.size(); i++) {
                    if(null == li.get(i) || li.get(i).isEmpty()){
                        continue;
                    }
                    List<Element> children = li.get(i).getChildElements();
                    JSONObject tranSub = new JSONObject();
                    JSONArray tranSubA = new JSONArray();
                    String key = null;
                    JSONArray value = new JSONArray();
                    for (int j = 0; j <children.size(); j++) {
                        Element child = children.get(j);
                        if(null== child || child.isEmpty()){
                            continue;
                        }
                        String type = child.getName();
                        if(type.equals("span")){
                            key = filterSpecialChar(child.getContent().toString());
                        }else if (type.equals("ul")){
                            value = getUlText(child);
                        }else if (type.equals("p")){
                            List<Element> pChildren = child.getChildElements();
                            if(null != pChildren && pChildren.size() > 0){
                                for (int k = 0; k < pChildren.size(); k++) {
                                    value.add(filterSpecialChar(pChildren.get(k).getContent().toString()));
                                }
                            }else {
                                value.add(filterSpecialChar(child.getContent().toString()));
                            }
                        }else if (type.equals("div")){
                            Element dspan = child.getFirstElement("span");
                            List<Element> dp = child.getAllElements("p");
                            if(null != dspan && !dspan.isEmpty()){
                                key = filterSpecialChar(dspan.getContent().toString());
                            }
                            if(null != dp && !dp.isEmpty()){
                                for (int k = 0; k < dp.size(); k++) {
                                    value.add(filterSpecialChar(dp.get(k).getContent().toString()));
                                }
                            }
                        }
                    }
                    if(null == key){
                        key = getText(li.get(i));
                        if(null == key || "".equals(key)){
                            continue;
                        }
                    }
                    if(null != value && value.size()>0){
                        tranSub.put(key,value);
                        UL.add(tranSub);
                    }else {
                        UL.add(key);
                    }

                }
            }
        }
        if(UL.isEmpty()){
            return null;
        }
        return UL;
    }

    public static String getElementText(Element el, String tag){
        String _ul = null;
        if(null == el || el.isEmpty()){
            return _ul;
        }
        _ul = el.getContent().toString();
        _ul = _ul.replaceAll("[\f,\n,\r,\t]", "");
        _ul = _ul.replaceAll(" {2,}", " ");
        _ul = _ul.replaceAll("&[a-zA-Z]{1,10};", " ");
        _ul = _ul.replaceAll("<[^/]+?>","");
        if(null != tag && !"".equals(tag)){
            _ul = _ul.replaceAll("</["+tag+"]+?>","");
        }
        _ul = _ul.replaceAll("<.+?>","\n");
        _ul = _ul.replaceAll("\\n *\\n","\n");
        return _ul;
    }

    public static String filterSpecialChar(String content){
        content = content.replaceAll("(?<=>)\\s+(?=<)", "");
        content = content.replaceAll("<.+?>","");
        content = content.replaceAll("[\f,\n,\r,\t]","");
        content = content.replaceAll(" {2,}", " ");
        content = content.replaceAll("&[a-zA-Z]{1,10};", " ");
        return content.trim();
    }

    public static String getText(Element el){
        String text = null;
        if(null != el && !el.isEmpty()){
            text = el.toString();
            String key = "<.+?>";
            Pattern p = Pattern.compile(key);
            Matcher m = p.matcher(text);
            int count = 0,startTag=0,start=0;
            String result = "";
            while (m.find()){
                String tag = m.group(0);
                if(start != 0){
                    result += text.substring(start,text.indexOf(tag,count));
                    start = 0;
                }
                if(tag.startsWith("</")){
                    startTag--;
                    if(startTag==1){
                        start = text.indexOf(tag,count) + tag.length();
                    }else if(startTag==0){
                        result += text.substring(count,text.indexOf(tag,count));
                    }
                }else{
                    startTag++;
                }
                count = text.indexOf(tag,count) + tag.length();
            }
            result = result.replaceAll("<[^/]+?>[\\s,\\S]*</.+?>", "");
            result = result.replaceAll("[\f,\n,\r,\t]","");
            result = result.replaceAll(" {2,}", " ");
            result = result.replaceAll("&[a-zA-Z]{1,10};", " ");
            result = result.trim();
            text = result;
        }
        return text;
    }

    public static String filterOffUtf8Mb4(String text) throws UnsupportedEncodingException {
        byte[] bytes = text.getBytes("utf-8");
        ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
        int i = 0;
        while (i < bytes.length) {
            short b = bytes[i];
            if (b > 0) {
                buffer.put(bytes[i++]);
                continue;
            }

            b += 256; // 去掉符号位

            if (((b >> 5) ^ 0x6) == 0) {
                buffer.put(bytes, i, 2);
                i += 2;
            } else if (((b >> 4) ^ 0xE) == 0) {
                buffer.put(bytes, i, 3);
                i += 3;
            } else if (((b >> 3) ^ 0x1E) == 0) {
                i += 4;
            } else if (((b >> 2) ^ 0x3E) == 0) {
                i += 5;
            } else if (((b >> 1) ^ 0x7E) == 0) {
                i += 6;
            } else {
                buffer.put(bytes[i++]);
            }
        }
        buffer.flip();
        return new String(buffer.array(), "utf-8");
    }

    public static void main1(String[] args) throws UnsupportedEncodingException {
        JSONObject content = YouDaoFile.parserHtml("D:\\youdao\\youdao\\zone-1449669101772");
        System.out.println(content.toString()+"\n");
        System.out.println(filterOffUtf8Mb4(content.toString())+"\n");
//        String word =  FileUtils.readFile("D:\\youdao.txt");
//        String[] words = word.split(",");
//        List<String> list = Arrays.asList(words);
//        String path = "D:\\youdao\\";
//        File file = new File(path);
//        String[] fileList = null;//{"nul-1449661114610"};
//        if(!file.exists() || !file.isDirectory()) {
//            System.out.println("文件读取错误");
//            return;
//        }
//        fileList = file.list();
//        int count = 0 ;
//        for (int i = 0; i < fileList.length; i++) {
//            if(fileList[i].indexOf("-") <= -1){
//                continue;
//            }
//            String fileName = fileList[i].substring(0,fileList[i].lastIndexOf("-"));
//            if(word.indexOf("\""+fileName+"\"")>-1){
//                System.out.println(fileName+" "+count++);
//            }

//        }
    }
    public static void main(String[] args) {
        String path = "D:\\youdao\\lost-20160202-1\\";
//        String path = "D:\\youdao\\lost-20160202\\";
//        String path = "D:\\youdao\\youdao\\";
        File file = new File(path);
        BufferedReader reader = null;
        String className = "com.mysql.jdbc.Driver";
//        String url = "jdbc:mysql://192.168.1.188:3306/ieltsdb?zeroDateTimeBehavior=convertToNull";
//        String username = "ieltsdba";
        String url = "jdbc:mysql://192.168.1.188:3306/xzdb?zeroDateTimeBehavior=convertToNull";
        String username = "xzdba";
        String userpass = "zaq1_XSW2";
        Connection con = null;
        //PreparedStatement pstmt = null;
        PreparedStatement stmt = null;
        ResultSet rs = null;
        try {
            String[] fileList = null;//{"nul-1449661114610"};
            if(!file.exists() || !file.isDirectory()) {
                System.out.println("文件读取错误");
                return;
            }
            fileList = file.list();
            Class.forName(className);
            con = DriverManager.getConnection(url, username, userpass);
            con.setAutoCommit(false);
            //pstmt = con.prepareStatement("select id from t_dictionary_temp1 where word =  binary(?)");
            stmt = con.prepareStatement("INSERT into t_dictionary_temp1(word,content) VALUES (?, ? );");
            //reader = new BufferedReader(new FileReader(file));
            //String line = null;
            int id = 0;
            boolean flag = false;
            //while ((line = reader.readLine()) != null) {
            for (int i = 0; i < fileList.length; i++) {
                String fileName = fileList[i];
                JSONObject json = null;
                try {
                    json = YouDaoFile.parserHtml(path + fileName);
                }catch (Exception e){
                    System.out.println(fileName);
                    continue;
                }
                if(json.get("state").equals("error")){
                    System.out.println(fileName + ":" + json.get("message"));
                    continue;
                }
                String name = fileName;//.substring(0,fileName.lastIndexOf("-"));
                stmt.setString(1,name);
                JSONObject content = null;
                if(json.containsKey("content")){
                    content = (JSONObject) json.get("content");
                    System.out.println(content);
                    stmt.setString(2, filterOffUtf8Mb4(content.toString()));
                }else {
                    stmt.setString(2,"");
                }
//                stmt.execute();
                stmt.addBatch();
                if(i%100==0){
                    stmt.executeBatch();
                    con.commit();
                    con.setAutoCommit(false);
                }
            }
             stmt.executeBatch();
            con.commit();
            con.close();
//            reader.close();
        } catch (java.lang.ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
    }
}
